# -*- coding: utf-8 -*-

# Define here the models for your spider middleware
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
import random
import time
import traceback

import requests
from scrapy import signals
from Banshan.utils.UserAgent import User_Agent_list
from scrapy.http import HtmlResponse

from twisted.internet import defer
from twisted.internet.error import TimeoutError, DNSLookupError, \
    ConnectionRefusedError, ConnectionDone, ConnectError, \
    ConnectionLost, TCPTimedOutError
from twisted.web.client import ResponseFailed
from scrapy.core.downloader.handlers.http11 import TunnelError
from scrapy.exceptions import IgnoreRequest
from scrapy import signals
from scrapy.utils.project import get_project_settings
import json
import logging
import requests
from twisted.internet.defer import DeferredLock
from Banshan.utils.ProxyModel import ProxyModel

settings = get_project_settings()
logger = logging.getLogger(__name__)





class IPProxyDownloadMiddleware(object):
    # 芝麻代理请求API
    PROXY_URL = 'http://webapi.http.zhimacangku.com/getip?num=1&type=2&pro=&city=0&yys=0&port=1&time=1&ts=1&ys=0&cs=0&lb=1&sb=0&pb=45&mr=1&regions='

    def __init__(self):
        super(IPProxyDownloadMiddleware, self).__init__()
        self.current_proxy = None
        self.lock = DeferredLock()

    def process_request(self, request, spider):
        if 'proxy' not in request.meta or self.current_proxy.is_expiring:
            # 请求代理
            self.update_proxy()

        request.meta['proxy'] = self.current_proxy.proxy

    def process_response(self, request, response, spider):
        if response.status not in [200, 301, 302] and request.meta.get('ip_opener'):
            if not self.current_proxy.blacked:
                self.current_proxy.blacked = True
            logger.error('%s 这个代理被加入黑名单了' % self.current_proxy.ip)
            self.update_proxy()
            return request
        return response

    def process_exception(self, request, exception, spider):
        msg = '下载异常:{}\n报错信息如下:{}'.format(request.url, exception)
        logger.error(msg)
        if '没有正确答复或连接的主机没有反应' in msg:
            self.update_proxy()
            request.meta['proxy'] = self.current_proxy.proxy
            logger.error('下载连接超时， 更新ip')
            return request

    def update_proxy(self):
        self.lock.acquire()
        if not self.current_proxy or self.current_proxy.is_expiring or self.current_proxy.blacked:
            response = requests.get(self.PROXY_URL)
            text = response.text
            logging.error('重新获取了一个代理:{}'.format(text))
            result = json.loads(text)
            if len(result['data']) > 0:
                data = result['data'][0]
                proxy_model = ProxyModel(data)
                self.current_proxy = proxy_model
        self.lock.release()


